{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import collections\n",
    "import math\n",
    "import os\n",
    "import random\n",
    "import sys\n",
    "import threading\n",
    "import time\n",
    "import zipfile\n",
    "\n",
    "import numpy as np\n",
    "from six.moves import urllib\n",
    "from six.moves import xrange  # pylint: disable=redefined-builtin\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.python.client import device_lib\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/gpu:0']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "local_device_protos = device_lib.list_local_devices()\n",
    "[x.name for x in local_device_protos if x.device_type == 'GPU']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "word2vec = tf.load_op_library('./word2vec_ops.so')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "flags = tf.app.flags\n",
    "\n",
    "flags.DEFINE_string(\"save_path\", None, \"Directory to write the model and \"\n",
    "                    \"training summaries.\")\n",
    "flags.DEFINE_string(\"train_data\", None, \"Training text file. \"\n",
    "                    \"E.g., unzipped file http://mattmahoney.net/dc/text8.zip.\")\n",
    "flags.DEFINE_string(\n",
    "    \"eval_data\", None, \"File consisting of analogies of four tokens.\"\n",
    "    \"embedding 2 - embedding 1 + embedding 3 should be close \"\n",
    "    \"to embedding 4.\"\n",
    "    \"See README.md for how to get 'questions-words.txt'.\")\n",
    "flags.DEFINE_integer(\"embedding_size\", 200, \"The embedding dimension size.\")\n",
    "flags.DEFINE_integer(\n",
    "    \"epochs_to_train\", 15,\n",
    "    \"Number of epochs to train. Each epoch processes the training data once \"\n",
    "    \"completely.\")\n",
    "flags.DEFINE_float(\"learning_rate\", 0.2, \"Initial learning rate.\")\n",
    "flags.DEFINE_integer(\"num_neg_samples\", 100,\n",
    "                     \"Negative samples per training example.\")\n",
    "flags.DEFINE_integer(\"batch_size\", 16,\n",
    "                     \"Number of training examples processed per step \"\n",
    "                     \"(size of a minibatch).\")\n",
    "flags.DEFINE_integer(\"concurrent_steps\", 12,\n",
    "                     \"The number of concurrent training steps.\")\n",
    "flags.DEFINE_integer(\"window_size\", 5,\n",
    "                     \"The number of words to predict to the left and right \"\n",
    "                     \"of the target word.\")\n",
    "flags.DEFINE_integer(\"min_count\", 5,\n",
    "                     \"The minimum number of word occurrences for it to be \"\n",
    "                     \"included in the vocabulary.\")\n",
    "flags.DEFINE_float(\"subsample\", 1e-3,\n",
    "                   \"Subsample threshold for word occurrence. Words that appear \"\n",
    "                   \"with higher frequency will be randomly down-sampled. Set \"\n",
    "                   \"to 0 to disable.\")\n",
    "flags.DEFINE_boolean(\n",
    "    \"interactive\", False,\n",
    "    \"If true, enters an IPython interactive session to play with the trained \"\n",
    "    \"model. E.g., try model.analogy(b'france', b'paris', b'russia') and \"\n",
    "    \"model.nearby([b'proton', b'elephant', b'maxwell'])\")\n",
    "flags.DEFINE_integer(\"statistics_interval\", 5,\n",
    "                     \"Print statistics every n seconds.\")\n",
    "flags.DEFINE_integer(\"summary_interval\", 5,\n",
    "                     \"Save training summary to file every n seconds (rounded \"\n",
    "                     \"up to statistics interval).\")\n",
    "flags.DEFINE_integer(\"checkpoint_interval\", 600,\n",
    "                     \"Checkpoint the model (i.e. save the parameters) every n \"\n",
    "                     \"seconds (rounded up to statistics interval).\")\n",
    "\n",
    "FLAGS = flags.FLAGS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Options(object):\n",
    "    \"\"\"Options used by our word2vec model.\"\"\"\n",
    "\n",
    "    def __init__(self):\n",
    "        # Model options.\n",
    "\n",
    "        # Embedding dimension.\n",
    "        self.emb_dim = FLAGS.embedding_size\n",
    "\n",
    "        # Training options.\n",
    "        # The training text file.\n",
    "        self.train_data = FLAGS.train_data\n",
    "\n",
    "        # Number of negative samples per example.\n",
    "        self.num_samples = FLAGS.num_neg_samples\n",
    "\n",
    "        # The initial learning rate.\n",
    "        self.learning_rate = FLAGS.learning_rate\n",
    "\n",
    "        # Number of epochs to train. After these many epochs, the learning\n",
    "        # rate decays linearly to zero and the training stops.\n",
    "        self.epochs_to_train = FLAGS.epochs_to_train\n",
    "\n",
    "        # Concurrent training steps.\n",
    "        self.concurrent_steps = FLAGS.concurrent_steps\n",
    "\n",
    "        # Number of examples for one training step.\n",
    "        self.batch_size = FLAGS.batch_size\n",
    "\n",
    "        # The number of words to predict to the left and right of the target word.\n",
    "        self.window_size = FLAGS.window_size\n",
    "\n",
    "        # The minimum number of word occurrences for it to be included in the\n",
    "        # vocabulary.\n",
    "        self.min_count = FLAGS.min_count\n",
    "\n",
    "        # Subsampling threshold for word occurrence.\n",
    "        self.subsample = FLAGS.subsample\n",
    "\n",
    "        # How often to print statistics.\n",
    "        self.statistics_interval = FLAGS.statistics_interval\n",
    "\n",
    "        # How often to write to the summary file (rounds up to the nearest\n",
    "        # statistics_interval).\n",
    "        self.summary_interval = FLAGS.summary_interval\n",
    "\n",
    "        # How often to write checkpoints (rounds up to the nearest statistics\n",
    "        # interval).\n",
    "        self.checkpoint_interval = FLAGS.checkpoint_interval\n",
    "\n",
    "        # Where to write out summaries.\n",
    "        self.save_path = FLAGS.save_path\n",
    "        if not os.path.exists(self.save_path):\n",
    "            os.makedirs(self.save_path)\n",
    "\n",
    "        # Eval options.\n",
    "        # The text file for eval.\n",
    "        self.eval_data = FLAGS.eval_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Word2Vec(object):\n",
    "    \"\"\"Word2Vec model (Skipgram).\"\"\"\n",
    "\n",
    "    def __init__(self, options, session):\n",
    "        self._options = options\n",
    "        self._session = session\n",
    "        self._word2id = {}\n",
    "        self._id2word = []\n",
    "        self.build_graph()\n",
    "        self.build_eval_graph()\n",
    "        self.save_vocab()\n",
    "\n",
    "    def read_analogies(self):\n",
    "        \"\"\"Reads through the analogy question file.\n",
    "        Returns:\n",
    "          questions: a [n, 4] numpy array containing the analogy question's\n",
    "                     word ids.\n",
    "          questions_skipped: questions skipped due to unknown words.\n",
    "        \"\"\"\n",
    "        questions = []\n",
    "        questions_skipped = 0\n",
    "        with open(self._options.eval_data, \"rb\") as analogy_f:\n",
    "            for line in analogy_f:\n",
    "                if line.startswith(b\":\"):  # Skip comments.\n",
    "                    continue\n",
    "                words = line.strip().lower().split(b\" \")\n",
    "                ids = [self._word2id.get(w.strip()) for w in words]\n",
    "\n",
    "                if None in ids or len(ids) != 4:\n",
    "                    questions_skipped += 1\n",
    "                else:\n",
    "                    questions.append(np.array(ids))\n",
    "\n",
    "        print(\"Eval analogy file: \", self._options.eval_data)\n",
    "        print(\"Questions: \", len(questions))\n",
    "        print(\"Skipped: \", questions_skipped)\n",
    "        self._analogy_questions = np.array(questions, dtype=np.int32)\n",
    "\n",
    "    def forward(self, examples, labels):\n",
    "        \"\"\"Build the graph for the forward pass.\"\"\"\n",
    "        opts = self._options\n",
    "\n",
    "        # Declare all variables we need.\n",
    "        # Embedding: [vocab_size, emb_dim]\n",
    "        init_width = 0.5 / opts.emb_dim\n",
    "        emb = tf.Variable(\n",
    "            tf.random_uniform(\n",
    "                [opts.vocab_size, opts.emb_dim], -init_width, init_width),\n",
    "            name=\"emb\")\n",
    "        self._emb = emb\n",
    "\n",
    "        # Softmax weight: [vocab_size, emb_dim]. Transposed.\n",
    "        sm_w_t = tf.Variable(\n",
    "            tf.zeros([opts.vocab_size, opts.emb_dim]),\n",
    "            name=\"sm_w_t\")\n",
    "\n",
    "        # Softmax bias: [vocab_size].\n",
    "        sm_b = tf.Variable(tf.zeros([opts.vocab_size]), name=\"sm_b\")\n",
    "\n",
    "        # Global step: scalar, i.e., shape [].\n",
    "        self.global_step = tf.Variable(0, name=\"global_step\")\n",
    "\n",
    "        # Nodes to compute the nce loss w/ candidate sampling.\n",
    "        labels_matrix = tf.reshape(\n",
    "            tf.cast(labels,\n",
    "                    dtype=tf.int64),\n",
    "            [opts.batch_size, 1])\n",
    "\n",
    "        # Negative sampling.\n",
    "        sampled_ids, _, _ = (tf.nn.fixed_unigram_candidate_sampler(\n",
    "            true_classes=labels_matrix,\n",
    "            num_true=1,\n",
    "            num_sampled=opts.num_samples,\n",
    "            unique=True,\n",
    "            range_max=opts.vocab_size,\n",
    "            distortion=0.75,\n",
    "            unigrams=opts.vocab_counts.tolist()))\n",
    "\n",
    "        # Embeddings for examples: [batch_size, emb_dim]\n",
    "        example_emb = tf.nn.embedding_lookup(emb, examples)\n",
    "\n",
    "        # Weights for labels: [batch_size, emb_dim]\n",
    "        true_w = tf.nn.embedding_lookup(sm_w_t, labels)\n",
    "        # Biases for labels: [batch_size, 1]\n",
    "        true_b = tf.nn.embedding_lookup(sm_b, labels)\n",
    "\n",
    "        # Weights for sampled ids: [num_sampled, emb_dim]\n",
    "        sampled_w = tf.nn.embedding_lookup(sm_w_t, sampled_ids)\n",
    "        # Biases for sampled ids: [num_sampled, 1]\n",
    "        sampled_b = tf.nn.embedding_lookup(sm_b, sampled_ids)\n",
    "\n",
    "        # True logits: [batch_size, 1]\n",
    "        true_logits = tf.reduce_sum(tf.multiply(example_emb, true_w), 1) + true_b\n",
    "\n",
    "        # Sampled logits: [batch_size, num_sampled]\n",
    "        # We replicate sampled noise labels for all examples in the batch\n",
    "        # using the matmul.\n",
    "        sampled_b_vec = tf.reshape(sampled_b, [opts.num_samples])\n",
    "        sampled_logits = tf.matmul(example_emb,\n",
    "                                   sampled_w,\n",
    "                                   transpose_b=True) + sampled_b_vec\n",
    "        return true_logits, sampled_logits\n",
    "\n",
    "    def nce_loss(self, true_logits, sampled_logits):\n",
    "        \"\"\"Build the graph for the NCE loss.\"\"\"\n",
    "\n",
    "        # cross-entropy(logits, labels)\n",
    "        opts = self._options\n",
    "        true_xent = tf.nn.sigmoid_cross_entropy_with_logits(\n",
    "            labels=tf.ones_like(true_logits), logits=true_logits)\n",
    "        sampled_xent = tf.nn.sigmoid_cross_entropy_with_logits(\n",
    "            labels=tf.zeros_like(sampled_logits), logits=sampled_logits)\n",
    "\n",
    "        # NCE-loss is the sum of the true and noise (sampled words)\n",
    "        # contributions, averaged over the batch.\n",
    "        nce_loss_tensor = (tf.reduce_sum(true_xent) +\n",
    "                           tf.reduce_sum(sampled_xent)) / opts.batch_size\n",
    "        return nce_loss_tensor\n",
    "\n",
    "    def optimize(self, loss):\n",
    "        \"\"\"Build the graph to optimize the loss function.\"\"\"\n",
    "\n",
    "        # Optimizer nodes.\n",
    "        # Linear learning rate decay.\n",
    "        opts = self._options\n",
    "        words_to_train = float(opts.words_per_epoch * opts.epochs_to_train)\n",
    "        lr = opts.learning_rate * tf.maximum(\n",
    "            0.0001, 1.0 - tf.cast(self._words, tf.float32) / words_to_train)\n",
    "        self._lr = lr\n",
    "        optimizer = tf.train.GradientDescentOptimizer(lr)\n",
    "        train = optimizer.minimize(loss,\n",
    "                                   global_step=self.global_step,\n",
    "                                   gate_gradients=optimizer.GATE_NONE)\n",
    "        self._train = train\n",
    "\n",
    "    def build_eval_graph(self):\n",
    "        \"\"\"Build the eval graph.\"\"\"\n",
    "        # Eval graph\n",
    "\n",
    "        # Each analogy task is to predict the 4th word (d) given three\n",
    "        # words: a, b, c.  E.g., a=italy, b=rome, c=france, we should\n",
    "        # predict d=paris.\n",
    "\n",
    "        # The eval feeds three vectors of word ids for a, b, c, each of\n",
    "        # which is of size N, where N is the number of analogies we want to\n",
    "        # evaluate in one batch.\n",
    "        analogy_a = tf.placeholder(dtype=tf.int32)  # [N]\n",
    "        analogy_b = tf.placeholder(dtype=tf.int32)  # [N]\n",
    "        analogy_c = tf.placeholder(dtype=tf.int32)  # [N]\n",
    "\n",
    "        # Normalized word embeddings of shape [vocab_size, emb_dim].\n",
    "        nemb = tf.nn.l2_normalize(self._emb, 1)\n",
    "\n",
    "        # Each row of a_emb, b_emb, c_emb is a word's embedding vector.\n",
    "        # They all have the shape [N, emb_dim]\n",
    "        a_emb = tf.gather(nemb, analogy_a)  # a's embs\n",
    "        b_emb = tf.gather(nemb, analogy_b)  # b's embs\n",
    "        c_emb = tf.gather(nemb, analogy_c)  # c's embs\n",
    "\n",
    "        # We expect that d's embedding vectors on the unit hyper-sphere is\n",
    "        # near: c_emb + (b_emb - a_emb), which has the shape [N, emb_dim].\n",
    "        target = c_emb + (b_emb - a_emb)\n",
    "\n",
    "        # Compute cosine distance between each pair of target and vocab.\n",
    "        # dist has shape [N, vocab_size].\n",
    "        dist = tf.matmul(target, nemb, transpose_b=True)\n",
    "\n",
    "        # For each question (row in dist), find the top 4 words.\n",
    "        _, pred_idx = tf.nn.top_k(dist, 4)\n",
    "\n",
    "        # Nodes for computing neighbors for a given word according to\n",
    "        # their cosine distance.\n",
    "        nearby_word = tf.placeholder(dtype=tf.int32)  # word id\n",
    "        nearby_emb = tf.gather(nemb, nearby_word)\n",
    "        nearby_dist = tf.matmul(nearby_emb, nemb, transpose_b=True)\n",
    "        nearby_val, nearby_idx = tf.nn.top_k(nearby_dist,\n",
    "                                             min(1000, self._options.vocab_size))\n",
    "\n",
    "        # Nodes in the construct graph which are used by training and\n",
    "        # evaluation to run/feed/fetch.\n",
    "        self._analogy_a = analogy_a\n",
    "        self._analogy_b = analogy_b\n",
    "        self._analogy_c = analogy_c\n",
    "        self._analogy_pred_idx = pred_idx\n",
    "        self._nearby_word = nearby_word\n",
    "        self._nearby_val = nearby_val\n",
    "        self._nearby_idx = nearby_idx\n",
    "\n",
    "    def build_graph(self):\n",
    "        \"\"\"Build the graph for the full model.\"\"\"\n",
    "        opts = self._options\n",
    "        # The training data. A text file.\n",
    "        (words, counts, words_per_epoch, self._epoch, self._words, examples,\n",
    "         labels) = word2vec.skipgram_word2vec(filename=opts.train_data,\n",
    "                                              batch_size=opts.batch_size,\n",
    "                                              window_size=opts.window_size,\n",
    "                                              min_count=opts.min_count,\n",
    "                                              subsample=opts.subsample)\n",
    "        (opts.vocab_words, opts.vocab_counts,\n",
    "         opts.words_per_epoch) = self._session.run([words, counts, words_per_epoch])\n",
    "        opts.vocab_size = len(opts.vocab_words)\n",
    "\n",
    "        print(\"Data file: \", opts.train_data)\n",
    "        print(\"Vocab size: \", opts.vocab_size - 1, \" + UNK\")\n",
    "        print(\"Words per epoch: \", opts.words_per_epoch)\n",
    "\n",
    "        self._examples = examples\n",
    "        self._labels = labels\n",
    "        self._id2word = opts.vocab_words\n",
    "\n",
    "        for i, w in enumerate(self._id2word):\n",
    "            self._word2id[w] = i\n",
    "\n",
    "        true_logits, sampled_logits = self.forward(examples, labels)\n",
    "        loss = self.nce_loss(true_logits, sampled_logits)\n",
    "        tf.summary.scalar(\"NCE loss\", loss)\n",
    "        self._loss = loss\n",
    "        self.optimize(loss)\n",
    "\n",
    "        # Properly initialize all variables.\n",
    "        tf.global_variables_initializer().run()\n",
    "\n",
    "        self.saver = tf.train.Saver()\n",
    "\n",
    "    def save_vocab(self):\n",
    "        \"\"\"Save the vocabulary to a file so the model can be reloaded.\"\"\"\n",
    "        opts = self._options\n",
    "\n",
    "        with open(os.path.join(opts.save_path, \"vocab.txt\"), \"w\") as f:\n",
    "\n",
    "            for i in xrange(opts.vocab_size):\n",
    "                vocab_word = tf.compat.as_text(opts.vocab_words[i]).encode(\"utf-8\")\n",
    "                f.write(\"%s %d\\n\" % (vocab_word,\n",
    "                                 opts.vocab_counts[i]))\n",
    "\n",
    "    def _train_thread_body(self):\n",
    "        initial_epoch, = self._session.run([self._epoch])\n",
    "        while True:\n",
    "            _, epoch = self._session.run([self._train, self._epoch])\n",
    "\n",
    "            if epoch != initial_epoch:\n",
    "                break\n",
    "\n",
    "    def train(self):\n",
    "        \"\"\"Train the model.\"\"\"\n",
    "        opts = self._options\n",
    "\n",
    "        initial_epoch, initial_words = self._session.run([self._epoch, self._words])\n",
    "\n",
    "        summary_op = tf.summary.merge_all()\n",
    "        summary_writer = tf.summary.FileWriter(opts.save_path, self._session.graph)\n",
    "        workers = []\n",
    "        for _ in xrange(opts.concurrent_steps):\n",
    "            t = threading.Thread(target=self._train_thread_body)\n",
    "            t.start()\n",
    "            workers.append(t)\n",
    "\n",
    "        last_words, last_time, last_summary_time = initial_words, time.time(), 0\n",
    "        last_checkpoint_time = 0\n",
    "\n",
    "        while True:\n",
    "            time.sleep(opts.statistics_interval)  # Reports our progress once a while.\n",
    "            (epoch, step, loss, words, lr) = self._session.run(\n",
    "              [self._epoch, self.global_step, self._loss, self._words, self._lr])\n",
    "\n",
    "            now = time.time()\n",
    "            last_words, last_time, rate = words, now, (words - last_words) / (\n",
    "              now - last_time)\n",
    "\n",
    "            print(\"Epoch %4d Step %8d: lr = %5.3f loss = %6.2f words/sec = %8.0f\\r\" %\n",
    "                (epoch, step, lr, loss, rate), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "\n",
    "            if now - last_summary_time > opts.summary_interval:\n",
    "                summary_str = self._session.run(summary_op)\n",
    "                summary_writer.add_summary(summary_str, step)\n",
    "                last_summary_time = now\n",
    "\n",
    "            if now - last_checkpoint_time > opts.checkpoint_interval:\n",
    "                self.saver.save(self._session,\n",
    "                            os.path.join(opts.save_path, \"model.ckpt\"),\n",
    "                            global_step=step.astype(int))\n",
    "                last_checkpoint_time = now\n",
    "\n",
    "            if epoch != initial_epoch:\n",
    "                break\n",
    "\n",
    "        for t in workers:\n",
    "            t.join()\n",
    "\n",
    "        return epoch\n",
    "\n",
    "    def _predict(self, analogy):\n",
    "        \"\"\"Predict the top 4 answers for analogy questions.\"\"\"\n",
    "        idx, = self._session.run([self._analogy_pred_idx], {\n",
    "            self._analogy_a: analogy[:, 0],\n",
    "            self._analogy_b: analogy[:, 1],\n",
    "            self._analogy_c: analogy[:, 2]\n",
    "        })\n",
    "        return idx\n",
    "\n",
    "    def eval(self):\n",
    "        \"\"\"Evaluate analogy questions and reports accuracy.\"\"\"\n",
    "\n",
    "        # How many questions we get right at precision@1.\n",
    "        correct = 0\n",
    "\n",
    "        try:\n",
    "            total = self._analogy_questions.shape[0]\n",
    "        except AttributeError as e:\n",
    "            raise AttributeError(\"Need to read analogy questions.\")\n",
    "\n",
    "        start = 0\n",
    "        while start < total:\n",
    "            limit = start + 2500\n",
    "            sub = self._analogy_questions[start:limit, :]\n",
    "            idx = self._predict(sub)\n",
    "            start = limit\n",
    "\n",
    "            for question in xrange(sub.shape[0]):\n",
    "                for j in xrange(4):\n",
    "                    if idx[question, j] == sub[question, 3]:\n",
    "                        # Bingo! We predicted correctly. E.g., [italy, rome, france, paris].\n",
    "                        correct += 1\n",
    "                        break\n",
    "                    elif idx[question, j] in sub[question, :3]:\n",
    "                        # We need to skip words already in the question.\n",
    "                        continue\n",
    "                    else:\n",
    "                        # The correct label is not the precision@1\n",
    "                        break\n",
    "        print()\n",
    "        print(\"Eval %4d/%d accuracy = %4.1f%%\" % (correct, total,\n",
    "                                                  correct * 100.0 / total))\n",
    "\n",
    "\n",
    "    def analogy(self, w0, w1, w2):\n",
    "        \"\"\"Predict word w3 as in w0:w1 vs w2:w3.\"\"\"\n",
    "        wid = np.array([[self._word2id.get(w, 0) for w in [w0, w1, w2]]])\n",
    "        idx = self._predict(wid)\n",
    "\n",
    "        for c in [self._id2word[i] for i in idx[0, :]]:\n",
    "            if c not in [w0, w1, w2]:\n",
    "                print(c)\n",
    "                break\n",
    "        print(\"unknown\")\n",
    "\n",
    "    def nearby(self, words, num=20):\n",
    "        \"\"\"Prints out nearby words given a list of words.\"\"\"\n",
    "        ids = np.array([self._word2id.get(x, 0) for x in words])\n",
    "        vals, idx = self._session.run(\n",
    "            [self._nearby_val, self._nearby_idx], {self._nearby_word: ids})\n",
    "        for i in xrange(len(words)):\n",
    "\n",
    "            print(\"\\n%s\\n=====================================\" % (words[i]))\n",
    "\n",
    "            for (neighbor, distance) in zip(idx[i, :num], vals[i, :num]):\n",
    "                print(\"%-20s %6.4f\" % (self._id2word[neighbor], distance))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def _start_shell(local_ns=None):\n",
    "    # An interactive shell is useful for debugging/development.\n",
    "    import IPython\n",
    "    user_ns = {}\n",
    "    \n",
    "    if local_ns:\n",
    "        user_ns.update(local_ns)\n",
    "        user_ns.update(globals())\n",
    "        \n",
    "    IPython.start_ipython(argv=[], user_ns=user_ns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def main(_):\n",
    "    \"\"\"Train a word2vec model.\"\"\"\n",
    "    \n",
    "    FLAGS.train_data = \"text8\"\n",
    "    FLAGS.eval_data = \"questions-words.txt\"\n",
    "    FLAGS.save_path=\"/tmp\"\n",
    "    \n",
    "    if not FLAGS.train_data or not FLAGS.eval_data or not FLAGS.save_path:\n",
    "        print(\"--train_data --eval_data and --save_path must be specified.\")\n",
    "        sys.exit(1)\n",
    "        \n",
    "    opts = Options()\n",
    "    \n",
    "    with tf.Graph().as_default(), tf.Session() as session:\n",
    "        with tf.device(\"/cpu:0\"):\n",
    "            model = Word2Vec(opts, session)\n",
    "            model.read_analogies() # Read analogy questions\n",
    "            \n",
    "        for _ in xrange(opts.epochs_to_train):\n",
    "            model.train()  # Process one epoch\n",
    "            model.eval()  # Eval analogies.\n",
    "            \n",
    "        # Perform a final save.\n",
    "        model.saver.save(session,\n",
    "                         os.path.join(opts.save_path, \"model.ckpt\"),\n",
    "                         global_step=model.global_step)\n",
    "        \n",
    "        if FLAGS.interactive:\n",
    "            # E.g.,\n",
    "            # [0]: model.analogy(b'france', b'paris', b'russia')\n",
    "            # [1]: model.nearby([b'proton', b'elephant', b'maxwell'])\n",
    "            _start_shell(locals())            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data file:  text8\n",
      "Vocab size:  71290  + UNK\n",
      "Words per epoch:  17005207\n",
      "INFO:tensorflow:Summary name NCE loss is illegal; using NCE_loss instead.\n",
      "Eval analogy file:  questions-words.txt\n",
      "Questions:  17827\n",
      "Skipped:  1717\n",
      "Epoch    1 Step  4715964: lr = 0.190 loss =   4.08 words/sec =    16828\n",
      "Eval 1797/17827 accuracy = 10.1%\n",
      "Epoch    1 Step  5258636: lr = 0.189 loss =   4.51 words/sec =    16877\r"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-40:\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/threading.py\", line 914, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/threading.py\", line 862, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-22-bf037ee0cc03>\", line 239, in _train_thread_body\n",
      "    _, epoch = self._session.run([self._train, self._epoch])\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 767, in run\n",
      "    run_metadata_ptr)\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 903, in _run\n",
      "    raise RuntimeError('Attempted to use a closed Session.')\n",
      "RuntimeError: Attempted to use a closed Session.\n",
      "\n",
      "Exception in thread Thread-43:\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/threading.py\", line 914, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/threading.py\", line 862, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-22-bf037ee0cc03>\", line 239, in _train_thread_body\n",
      "    _, epoch = self._session.run([self._train, self._epoch])\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 767, in run\n",
      "    run_metadata_ptr)\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 903, in _run\n",
      "    raise RuntimeError('Attempted to use a closed Session.')\n",
      "RuntimeError: Attempted to use a closed Session.\n",
      "Exception in thread Thread-32:\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/threading.py\", line 914, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/threading.py\", line 862, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-22-bf037ee0cc03>\", line 239, in _train_thread_body\n",
      "    _, epoch = self._session.run([self._train, self._epoch])\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 767, in run\n",
      "    run_metadata_ptr)\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 903, in _run\n",
      "    raise RuntimeError('Attempted to use a closed Session.')\n",
      "RuntimeError: Attempted to use a closed Session.\n",
      "Exception in thread Thread-37:\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/threading.py\", line 914, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/threading.py\", line 862, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-22-bf037ee0cc03>\", line 239, in _train_thread_body\n",
      "    _, epoch = self._session.run([self._train, self._epoch])\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 767, in run\n",
      "    run_metadata_ptr)\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 903, in _run\n",
      "    raise RuntimeError('Attempted to use a closed Session.')\n",
      "RuntimeError: Attempted to use a closed Session.\n",
      "Exception in thread Thread-42:\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/threading.py\", line 914, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/threading.py\", line 862, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-22-bf037ee0cc03>\", line 239, in _train_thread_body\n",
      "    _, epoch = self._session.run([self._train, self._epoch])\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 767, in run\n",
      "    run_metadata_ptr)\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 903, in _run\n",
      "    raise RuntimeError('Attempted to use a closed Session.')\n",
      "RuntimeError: Attempted to use a closed Session.\n",
      "Exception in thread Thread-33:\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 1022, in _do_call\n",
      "    return fn(*args)\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 1004, in _run_fn\n",
      "    status, run_metadata)\n",
      "  File \"/usr/lib/python3.5/contextlib.py\", line 66, in __exit__\n",
      "    next(self.gen)\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/framework/errors_impl.py\", line 466, in raise_exception_on_not_ok_status\n",
      "    pywrap_tensorflow.TF_GetCode(status))\n",
      "tensorflow.python.framework.errors_impl.CancelledError: Run call was cancelled\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/threading.py\", line 914, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/threading.py\", line 862, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-22-bf037ee0cc03>\", line 239, in _train_thread_body\n",
      "    _, epoch = self._session.run([self._train, self._epoch])\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 767, in run\n",
      "    run_metadata_ptr)\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 965, in _run\n",
      "    feed_dict_string, options, run_metadata)\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 1015, in _do_run\n",
      "    target_list, options, run_metadata)\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 1035, in _do_call\n",
      "    raise type(e)(node_def, op, message)\n",
      "tensorflow.python.framework.errors_impl.CancelledError: Run call was cancelled\n",
      "Exception in thread Thread-41:\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/threading.py\", line 914, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/threading.py\", line 862, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-22-bf037ee0cc03>\", line 239, in _train_thread_body\n",
      "    _, epoch = self._session.run([self._train, self._epoch])\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 767, in run\n",
      "    run_metadata_ptr)\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 903, in _run\n",
      "    raise RuntimeError('Attempted to use a closed Session.')\n",
      "RuntimeError: Attempted to use a closed Session.\n",
      "Exception in thread Thread-36:\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/threading.py\", line 914, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/threading.py\", line 862, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-22-bf037ee0cc03>\", line 239, in _train_thread_body\n",
      "    _, epoch = self._session.run([self._train, self._epoch])\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 767, in run\n",
      "    run_metadata_ptr)\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 903, in _run\n",
      "    raise RuntimeError('Attempted to use a closed Session.')\n",
      "RuntimeError: Attempted to use a closed Session.\n",
      "Exception in thread Thread-39:\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/threading.py\", line 914, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/threading.py\", line 862, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-22-bf037ee0cc03>\", line 239, in _train_thread_body\n",
      "    _, epoch = self._session.run([self._train, self._epoch])\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 767, in run\n",
      "    run_metadata_ptr)\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 903, in _run\n",
      "    raise RuntimeError('Attempted to use a closed Session.')\n",
      "RuntimeError: Attempted to use a closed Session.\n",
      "Exception in thread Thread-35:\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 1022, in _do_call\n",
      "    return fn(*args)\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 1004, in _run_fn\n",
      "    status, run_metadata)\n",
      "  File \"/usr/lib/python3.5/contextlib.py\", line 66, in __exit__\n",
      "    next(self.gen)\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/framework/errors_impl.py\", line 466, in raise_exception_on_not_ok_status\n",
      "    pywrap_tensorflow.TF_GetCode(status))\n",
      "tensorflow.python.framework.errors_impl.CancelledError: Run call was cancelled\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/threading.py\", line 914, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/threading.py\", line 862, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-22-bf037ee0cc03>\", line 239, in _train_thread_body\n",
      "    _, epoch = self._session.run([self._train, self._epoch])\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 767, in run\n",
      "    run_metadata_ptr)\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 965, in _run\n",
      "    feed_dict_string, options, run_metadata)\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 1015, in _do_run\n",
      "    target_list, options, run_metadata)\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 1035, in _do_call\n",
      "    raise type(e)(node_def, op, message)\n",
      "tensorflow.python.framework.errors_impl.CancelledError: Run call was cancelled\n",
      "Exception in thread Thread-38:\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 1022, in _do_call\n",
      "    return fn(*args)\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 1004, in _run_fn\n",
      "    status, run_metadata)\n",
      "  File \"/usr/lib/python3.5/contextlib.py\", line 66, in __exit__\n",
      "    next(self.gen)\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/framework/errors_impl.py\", line 466, in raise_exception_on_not_ok_status\n",
      "    pywrap_tensorflow.TF_GetCode(status))\n",
      "tensorflow.python.framework.errors_impl.CancelledError: Run call was cancelled\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/threading.py\", line 914, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/threading.py\", line 862, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-22-bf037ee0cc03>\", line 239, in _train_thread_body\n",
      "    _, epoch = self._session.run([self._train, self._epoch])\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 767, in run\n",
      "    run_metadata_ptr)\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 965, in _run\n",
      "    feed_dict_string, options, run_metadata)\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 1015, in _do_run\n",
      "    target_list, options, run_metadata)\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 1035, in _do_call\n",
      "    raise type(e)(node_def, op, message)\n",
      "tensorflow.python.framework.errors_impl.CancelledError: Run call was cancelled\n",
      "Exception in thread Thread-34:\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/threading.py\", line 914, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/threading.py\", line 862, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<ipython-input-22-bf037ee0cc03>\", line 239, in _train_thread_body\n",
      "    _, epoch = self._session.run([self._train, self._epoch])\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 767, in run\n",
      "    run_metadata_ptr)\n",
      "  File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 903, in _run\n",
      "    raise RuntimeError('Attempted to use a closed Session.')\n",
      "RuntimeError: Attempted to use a closed Session.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m--------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-33-f72f060252e3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# sys.argv = [sys.argv[0], 'argument1', 'argument2', 'argument2']\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/platform/app.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(main, argv)\u001b[0m\n\u001b[1;32m     42\u001b[0m   \u001b[0;31m# Call the main function, passing through any arguments\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     43\u001b[0m   \u001b[0;31m# to the final program.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 44\u001b[0;31m   \u001b[0m_sys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_sys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margv\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mflags_passthrough\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     45\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-32-b1f118d52e08>\u001b[0m in \u001b[0;36mmain\u001b[0;34m(_)\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0m_\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mxrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopts\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepochs_to_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m             \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# Process one epoch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     21\u001b[0m             \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# Eval analogies.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-22-bf037ee0cc03>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    260\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    261\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 262\u001b[0;31m             \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopts\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatistics_interval\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# Reports our progress once a while.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    263\u001b[0m             (epoch, step, loss, words, lr) = self._session.run(\n\u001b[1;32m    264\u001b[0m               [self._epoch, self.global_step, self._loss, self._words, self._lr])\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# sys.argv = [sys.argv[0], 'argument1', 'argument2', 'argument2']\n",
    "tf.app.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
